from multiprocessing.managers import BaseManager
from multiprocessing import Queue, Process
from DataOutput import DataOutput
from UrlManager import UrlManager
import time


class NodeManager:
    def start_manager(self, url_q, result_q):
        """
        创建一个分布式管理器
        :param url_q: url 队列
        :param result_q: 结果队列
        :return: BaseManager
        """
        # 把创建的两个队列注册在网络上，利用 register 方法，callable 参数关联了 Queue 对象
        # 将 Queue 对象在网络中暴露
        BaseManager.register('get_task_queue', callable=lambda:url_q)
        BaseManager.register('get_result_queue', callable=lambda:result_q)
        # 绑定端口 8001，设置验证口令"douban"，相当于对象的初始化并返回
        return BaseManager(address=('', 8001), authkey='douban'.encode('utf-8'))

    def url_manager_proc(self, url_q, conn_q, root_url):
        """
        url 管理进程
        :param url_q: url 队列
        :param conn_q: 解析得到的 url 队列
        :param root_url: 起始 url
        :return: None
        """
        url_manage = UrlManager()
        url_manage.add_new_url(root_url)
        while True:
            while url_manage.has_new_url():
                print('old_urls={}'.format(url_manage.old_urls_size()))
                new_url = url_manage.get_new_url()
                url_q.put(new_url)
                urls = conn_q.get()
                url_manage.add_new_urls(urls)
            else:
                url_q.put('end')
                print('控制节点发起结束通知')
                url_manage.save_progress('old_urls.txt', url_manage.old_urls)
                url_manage.save_progress('new_urls.txt', url_manage.new_urls)
                return

    def result_solve_proc(self, result_q, conn_q, store_q):
        """
        数据提取进程
        :param result_q: 未处理数据队列
        :param conn_q: 解析得到的 url 队列
        :param store_q: 解析后的数据队列
        :return:
        """
        while True:
            try:
                if not result_q.empty():
                    content = result_q.get()
                    if content['new_urls'] == 'end':
                        print('结果分析进程接收通知然后结束')
                        store_q.put('end')
                        return

                    conn_q.put(content['new_urls'])
                    store_q.put(content['data'])
                else:
                    time.sleep(0.1)
            except:
                time.sleep(0.1)

    def store_proc(self, store_q):
        """
        数据存储进程
        :param store_q: 解析后的数据队列
        :return:
        """
        output = DataOutput()
        while True:
            if not store_q.empty():
                data = store_q.get()

                if data == 'end':
                    print('存储进程接收结束通知然后结束')
                    return

                for item in data:
                    output.output_csv(item)
            else:
                time.sleep(0.1)


if __name__ == '__main__':
    # 初始化 4 个队列
    url_q = Queue()
    result_q = Queue()
    conn_q = Queue()
    store_q = Queue()
    # 创建分布式管理器
    node = NodeManager()
    manager = node.start_manager(url_q, result_q)
    # 创建 url 管理进程、数据提取进程和数据存储进程
    url = 'https://movie.douban.com/top250?start=0'
    url_manager_proc = Process(target=node.url_manager_proc, args=(url_q, conn_q, url,))
    result_solve_proc = Process(target=node.result_solve_proc, args=(result_q, conn_q, store_q,))
    store_proc = Process(target=node.store_proc, args=(store_q,))
    # 启动 3 个进程和分布式管理器
    url_manager_proc.start()
    result_solve_proc.start()
    store_proc.start()
    manager.get_server().serve_forever()



































